/*
 * Copyright (C) 2008 xyzse
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.

 * You should have received a copy of the GNU General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
*/

/**
 * @file XForwardIndex.cc
 * @version 0.0.0
 * @author zouxin ( zouxin2008@gmail.com )
 * @date 18/3/2008 0.0.0 created, by zouxin
 */
#include "XForwardIndex.h"
#include "StrTool.h"
#include "pws.h"
#include "config.h"
#include <stdlib.h>
#include <exception>
#include <stdexcept>

//#include "debug.h"

XForwardIndex::XForwardIndex(string rawfile,string docindexfile,string outputfile)
{
	m_inputRaw.open(rawfile.c_str(),ios::binary);
	if(!m_inputRaw)
	{
		cerr<<"can not open "<<rawfile<<endl;
		exit(-1);
	}

	m_inputDocIndex.open(docindexfile.c_str(),ios::binary);
	if(!m_inputDocIndex)
	{
		cerr<<"can not open "<<docindexfile<<endl;
		exit(-1);
	}


	m_output.open(outputfile.c_str(),ios::binary|ios::out);
	if(!m_output)
	{
		cerr<<"can not open "<<outputfile<<endl;
		exit(-1);
	}
	
}

XForwardIndex::~XForwardIndex()
{
	m_inputRaw.close();
	m_output.close();
	m_output.close();
}

int XForwardIndex::beginIndex()
{
	string templine;
	char cdocid[100],docoffset[100],url[256],md5[35];
	int docid;
	streampos offset_cur,offset_next=0,offset_save;
	streampos length;//content length
	int endflag=false;
	while(getline(m_inputDocIndex,templine))
	{
		offset_save=m_inputDocIndex.tellg();
		sscanf(templine.c_str(),"%s\t%s\t%s\t%s",cdocid,docoffset,url,md5);
		docid=atoi(cdocid);
		offset_cur=atoi(docoffset);
		
		if(getline(m_inputDocIndex,templine))
		{
			//get the next line
			sscanf(templine.c_str(),"%s\t%s\t%s\t%s",cdocid,docoffset,url,md5);
			offset_next=atoi(docoffset);
			m_inputDocIndex.seekg(offset_save,ios_base::beg);//restore previous line

			
			length=offset_next-offset_cur;
					
		}else
		{
			length=m_inputDocIndex.tellg()-offset_cur;
			endflag=true;
		}
	

		m_inputRaw.seekg(offset_cur,ios_base::beg);//raw data jump 
		///////////////read content /////////////////
		//cout<<docid<<"   "<<length<<endl;
		string content;
		if(endflag==true)
		{
			while(getline(m_inputRaw,templine))
				content+=templine;
			//segment(docid,content);
		}else
		{
			char ch;
			for(int i=0;i<length;i++)
			{
				m_inputRaw.get(ch);
				content+=ch;
			}
		}
	//	if(docid==151)cout<<content<<endl;
		try{
			segment(docid,content);
		    cout<<"segment "<<docid<<endl;
		}
		//catch(exception)
		catch(runtime_error )
		{
			cerr<<"catch exception when parsing "<<docid<<endl;
		}
		/////////////////////////////
	}//end while
	return 0;

}

void XForwardIndex::segment(int docid,string &content)
{
	//erase header and get body
	string::size_type pos;
	int newline=0;
	if((pos=content.find("\r\n\r\n"))!=string::npos)
		content=content.substr(pos+4);
	
	//del content between <script></script> and <style></style>
	StrTool::delTagContent(content,"script");
	StrTool::delTagContent(content,"SCRIPT");
	StrTool::delTagContent(content,"style");
	StrTool::delTagContent(content,"STYLE");
	//del all the tag <>
	StrTool::removeTags(content);
	//replace &nbsp with space
	StrTool::replaceStr(content,"&nbsp"," ");
	//del all the \t\r\n
	StrTool::emptyStr(content);
#ifdef debug
	if(docid==10)
	{
		//cout<<"!!!!!!!!!!!!!!!!!"<<docid<<endl;
		cout<<content;
#endif
	//////////////////word segment////////////////////////////////// 
    long total = 0, i = 0, * wordid = 0; 
    const char ** words = 0; 

    pws_result_t * pws_res = pws_res_create();
    if ( pws_res == 0 )
    {
        cerr<<"create word splitter object failed."<<endl;
        return ;
    }

    if ( pws_initialize(DIC_FILE.c_str()) != 0 )
    {
        pws_res_free( &pws_res );
        cerr<<"wordsplitter initialization failed."<<endl;
        return ;
    }

    {
        pws_segment_full( content.c_str(), content.length(), pws_res ); 
		total = pws_res_words( pws_res, &wordid, &words ); 

		m_output<<docid<<"\n";
		m_output<<total<<"\n";
        for ( i=0; i<total; i++ )
		{
			//fprintf( stdout, "%s/", *(words+i));
			m_output<<*(words+i)<<"/";
		}
		m_output<<"\n\n";
		        //fprintf( stdout, "\n" );
        pws_res_clean( pws_res );
	}
    
    pws_res_free( &pws_res ); 
    pws_release();
#ifdef debug
	}
#endif
}
